package com.zdb.demo.spark.stage;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.TaskScheduler;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class SparkStageTest {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setMaster("local[4]").setAppName("test");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        List<Long> list = new ArrayList<>();
        for (long i = 0L; i < 100L; i++) {
            list.add(i);
        }

        JavaPairRDD<Long, Long> cnt = jsc.parallelize(list, 4).mapToPair((Long aLong) -> {
            return new Tuple2<Long, Long>(aLong % 10, aLong);
        }).aggregateByKey(0L, new Function2<Long, Long, Long>() {
            @Override public Long call(Long v1, Long v2) throws Exception {
                return v1 + v2;
            }
        }, new Function2<Long, Long, Long>() {
            @Override public Long call(Long v1, Long v2) throws Exception {
                return v1 + v2;
            }
        });

        cnt.mapToPair(new PairFunction<Tuple2<Long, Long>, Long, Long>() {
            @Override public Tuple2<Long, Long> call(Tuple2<Long, Long> t2)
                    throws Exception {
                return new Tuple2<>(t2._1 % 5, t2._2);
            }
        }).groupByKey().foreach(new VoidFunction<Tuple2<Long, Iterable<Long>>>() {
            @Override public void call(Tuple2<Long, Iterable<Long>> t2)
                    throws Exception {
                System.out.println(t2._1 + "," + t2._2);
            }
        });
    }
}
